# -*- coding: utf-8 -*-
import time
import scrapy
import logging
import requests

from  .util.log import logging_util
from .util.scrapy_util_juejin import juejin_post_publish
from .util.scrapy_util_juejin import juejin_post_update
from .util.scrapy_util_juejin import test_juejin_api_getallPosts
from .util.scrapy_util_juejin import test_wp_category_add
from .util.scrapy_util_juejin import test_wp_tag_add
from .util.scrapy_util_xmind import xmind_generator_title_url
from .util.wordpress_util import wp_api_getallCategories
from .util.wordpress_util import wp_api_getallLabeles




# 日志模块
logging_util()
# 爬虫参数
global scrapy_from_xmind, xmind_filename, single_url

scrapy_from_xmind = True
xmind_filename = r'C:\Users\22934\Documents\我的导图\分布式.xmind'
single_url = 'https://juejin.im/post/6875179678441668615'
postid_dict = {}
directory = ''
label = ''
categoryExists = False
existLabel = False

label_array_push = []
# 爬博主全部文章系列。进行内容聚合
class JuejinSpiderSpider(scrapy.Spider):
    name = 'juejin_spider'
    allowed_domains = ['juejin.im']
    start_urls = []
    post_category = -1


    if(scrapy_from_xmind):
        # 掘金、
        totaldata_array = xmind_generator_title_url(xmind_filename)
        print(totaldata_array)
        directory = totaldata_array['directory']

        dict_categories = wp_api_getallCategories()
        for category in dict_categories.keys():
            if directory == category:
                categoryExists = True
                break
        if categoryExists == False:
            # create category
            data_return = test_wp_category_add(directory)
            post_category = data_return['id']
        else:
            post_category = dict_categories[directory]

        # label create
        label = totaldata_array['label']
        custom_labeles = label.split(' ')
        all_labeles = wp_api_getallLabeles()
        for label_custom in custom_labeles:
            existLabel = False
            for label in all_labeles.keys():
                if label == label_custom:
                    existLabel = True
                    label_array_push.append(all_labeles[label])
                    break
            if existLabel == False:
                labelid = test_wp_tag_add(label_custom)
                label_array_push.append(labelid['id'])


        # 什么样的链接放入爬虫后续：1、在文章库中没有的链接、
        # 文章库中有但是需要更新的内容进入更新流程，先进入更新流程，然后再进入爬虫过程

        postid = ''
        pages = 2
        page = 1
        per_page = 10
        category = []
        key_erro = -1

        for post_xmind in totaldata_array['totaldata_array']:
            if (post_xmind.link != ''):
                for page in range(1,pages):
                    obj_juejin = []
                    obj_juejin = test_juejin_api_getallPosts(page,per_page)
                    pages = int(obj_juejin['pages'])
                    page_data = obj_juejin['page_data']
                    for juejin_post in page_data:
                        juejin_post_link = juejin_post['content']['rendered'].split('\n')[-2].split('：')[1][0:-4]
                        logging.info("============114提取的链接============")
                        logging.info(juejin_post_link)
                        print(juejin_post_link)
                        if(juejin_post_link==post_xmind.link):
                            postid = juejin_post['id']
                            break;
                    if postid != '':
                        print(postid)
                        #response = juejin_post_update(category,id) # 更新操作
                        postid_dict[post_xmind.link] = postid
                    else:
                        postid_dict[post_xmind.link] = key_erro
                    postid_dict['category'] = post_category
                    start_urls.append(post_xmind.link)





    def parse(self, response):
        # 掘金 | 返回爬取到的内容  替换 data-original | 限制爬虫频次
        time.sleep(5)
        postid = postid_dict[response.url]
        post_category = postid_dict['category']
        if(postid>0):
            response = juejin_post_update(response,str(postid),[post_category],label_array_push)
            logging.info("==================update=====================")
        else:
            logging.info(response)
            category = [post_category]  # 1 | java基础源码研究   2 | 最近文章    3 | linux   4 | 数据库  5 | 微信收藏     6 | 代码实验室    7 | 面试官  8 | 开发日常
            response = juejin_post_publish(response, category,label_array_push)
            logging.info("==================publish=====================")
        logging.info(response.status_code)
        logging.info(response.content)
        pass
